﻿using System;
using System.Collections;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Text.RegularExpressions;
using System.Windows.Forms;
using System.IO;
using System.Drawing.Printing;
using System.Collections.Generic;
using smartBrowser.Ontology;
using System.Xml.XPath;
using System.Xml;

namespace smartBrowser.Util
{
    /// <summary>
    /// @author Joel
    /// </summary>  
    public class WebSite
    {
        
        private HtmlDocument document;
        private Uri url;
        private List<Ontology.Ontology> ontologyList;
        private String pageSourceCode;
        private Hashtable keywords;
        private Hashtable concepts;

        public List<Ontology.Ontology> OntologyList
        {
            get { return this.ontologyList; }            
        }

        public WebSite(HtmlDocument doc, String pageSourceCode)
        {                        
            this.document = doc;
            this.url = doc.Url;
            this.ontologyList = new List<Ontology.Ontology>();
            this.pageSourceCode = pageSourceCode;
            this.keywords = new Hashtable();
            this.concepts = new Hashtable();
        }

        public List<Tag> TopKeywords
        {
            get
            {
                return this.mostRankKeywords(this.keywords, 5);
            }
        }

        public List<Tag> TopConcepts
        {
            get
            {
                return this.mostRankKeywords(this.concepts, 5);
            }
        }

        public HtmlDocument SiteDocument
        {
            get 
            {
                return this.document; 
            }
        }

        public Uri Url
        {
            get { return this.url; }
        }

        public Hashtable KeyWords
        {
            get { return this.keywords; }
        }

        public Hashtable Concepts
        {
            get { return this.concepts; }
        }

        public String Source
        {
            get { return this.pageSourceCode; }
        }

        public String Title
        {
            get
            { 
                Regex regex = new Regex("<title>(.*?)</title>");
                return regex.Match(this.Source).Groups[1].Value;
            }
        }

        /// <summary>
        /// Faz parsing ao documento html
        /// </summary>
        /// <param name="headingFormat">recebe a string com o nome do elemento pelo qual quer fazer parsing</param>
        /// <returns>retorna o texto dessa tag</returns>
        private List<String> parseCommonTags(String headingFormat)
        {
            List<String> tags = new List<String>();

            HtmlElement body = document.Body;

            HtmlElementCollection collection = body.GetElementsByTagName(headingFormat);

            foreach (HtmlElement element in collection)
            {
                tags.Add(element.InnerText);
            }

            return tags;
        }

        /// <summary>
        /// tem definidas as tags pelo qual queremos fazer parsing, e chama o parseCommonTags
        /// </summary>
        /// <returns>retorna um array de strings com as palavras</returns>
        public List<Tag> findHTMLBuzzWords()
        {
            List<String> phrases = new List<String>();

            // title
            phrases.Add(this.document.Title);
            
            // h1 heading
            phrases.AddRange(parseCommonTags("h1"));            
            
            // h2 heading
            phrases.AddRange(parseCommonTags("h2"));

            // h3 heading
            phrases.AddRange(parseCommonTags("h3"));            

            // strong
            phrases.AddRange(parseCommonTags("strong"));
            phrases.AddRange(parseCommonTags("b"));

            // links
            phrases.AddRange(parseCommonTags("a"));            
            
            List<Tag> allTags = new List<Tag>();

            string[] words = null;

            foreach (string phrase in phrases)
            {
                if (phrase != null)
                {
                    words = phrase.Split(' ');

                    foreach (string word in words)
                    {
                        Tag tag = new Tag(null, word.Trim().ToLower());

                        if (word != null && !allTags.Contains(tag) && word.Length > 3)                                                    
                            allTags.Add(tag);
                        
                    }
                }
            }          

            return allTags;
        }

        /// <summary>
        /// Elimina a ocorrência de stopWords na lista de tags
        /// </summary>
        /// <param name="tags">a lista de tags</param>
        /// <returns>retorna a lista de tags sem as ocorrência das stopWords</returns>
        public List<Tag> readStopWords(List<Tag> tags)
        {            
            String en_stop_words = global::smartBrowser.Properties.Resources.EN_STOP_WORDS;
            String pt_stop_words = global::smartBrowser.Properties.Resources.PT_STOP_WORDS;

            String[] en_words = en_stop_words.Split('\n');
            String[] pt_words = pt_stop_words.Split('\n');
            
            List<String> stopWords = new List<string>();
            stopWords.AddRange(en_words);
            stopWords.AddRange(pt_words);

            List<Tag> tagsWithoutStopWords = new List<Tag>();

            for(int i = 0; i < stopWords.Count; i++)
            {
                stopWords[i] = stopWords[i].Replace('\r', ' ');
                stopWords[i] = stopWords[i].Trim().ToLower();
            }

            foreach(Tag tag in tags)
                if (!stopWords.Contains((String)tag.Concept.Trim().ToLower()))                
                    tagsWithoutStopWords.Add(tag);                                    

            return tagsWithoutStopWords;
        }

        /// <summary>
        /// Faz match da lista de strings com a ontologia local 
        /// </summary>
        /// <param name="tags">Lista de strings com as tags para fazer match</param>
        /// <param name="localOntology"> ontologia local do smart browser</param>
        /// <returns></returns>
        public void matchTags(List<Tag> tags, Ontology.Ontology localOntology)
        {
            Hashtable concepts = this.concepts;

            foreach (Tag tag in tags)
                if(localOntology.Ontologia.ContainsKey(tag.Concept.ToLower())){
                    this.keywords.Remove(tag.Concept.ToLower());                    
                    Tag conceptInOnology = (Tag)localOntology.Ontologia[tag.Concept.ToLower()];
                    conceptInOnology.Occurence = tag.Occurence;
                    if(!concepts.ContainsKey(tag.Concept.ToLower()))
                        concepts.Add(conceptInOnology.Concept.ToLower(), conceptInOnology);
                }

            this.concepts = concepts;
        }

        /// <summary>
        /// Verifica a ocorrência de cada tag no site
        /// </summary>
        /// <param name="wordList">Lista de strings</param>
        /// <returns>hashtable com a palavra como key, e um int com o numero de ocorrências dessa palavra no html document</returns>
        public void rankByOccurence(List<Tag> tagList)
        {
            Hashtable wordOccurence = this.keywords;            
                        
            Char[] specialChars = {'.', '$',  '^', '{', '[', '(', '|', ')', '*', '+', '?', '\\' };

            
            try
            {
                foreach (Tag tag in tagList)
                {
                    /*if (nav.Matches(tag.Concept.ToLower()))
                        MessageBox.Show("Encontrou a palavra: "+tag.Concept.ToLower());
                    */

                    String expression = tag.Concept.ToLower();

                    foreach (Char c in specialChars)
                         expression = expression.Replace("" + c, "\\" + c);

                    //de vez enkaundo dá erro, pk tipo qd a palavra tem caracteres especiais(por exemplo "c++"), ele atrofia com isso
                    Regex regex = new Regex(expression);

                    MatchCollection matches = regex.Matches(this.pageSourceCode.ToLower());

                    //MessageBox.Show("Word to filter: " + word+" occure "+matches.Count+" times");

                    if (matches.Count > 0 && !wordOccurence.ContainsKey(tag.Concept.ToLower()))
                    {
                        tag.Occurence = matches.Count;
                        wordOccurence.Add(tag.Concept.ToLower(), tag);
                    }

                }
            }catch(Exception){
            }
            
            this.keywords = wordOccurence;
           
        }

        /// <summary>
        /// Identifica as 10 melhores palavras do html document
        /// </summary>
        /// <param name="wordOccurence">recebe a hashtable com as ocorrências</param>
        /// <returns>retorna uma lista com as x melhores palavras</returns>
        public List<Tag> mostRankKeywords(Hashtable keywords, int top)
        {
            //Adicona as palavras com maior rank num array List (algoritmo pesadito..)
            List<Tag> topWords = new List<Tag>();            

            int count = 0;
            //Enquanto ainda não tiver seleccionado as top tags mais ocorridas,
            //ou se a hashtable ficou sem keys antes de se chegar às 10 tags
            while ((count <= top && count < keywords.Count) && keywords.Count != 0)
            {
                int mostRanked = 1;
                Tag wordTemp = null;

                //percorre a hashtable
                foreach (String word in keywords.Keys)
                {

                    int actualRanked = ((Tag)keywords[word]).Occurence;

                    //Se a key actual tiver maior ranking, passa a key para uma variável, e actualiza o maior ranking encontrado até agora
                    if (actualRanked >= mostRanked)
                    {
                        if (!topWords.Contains((Tag)keywords[word]))
                        {
                            wordTemp = (Tag)keywords[word];
                            mostRanked = actualRanked;
                        }
                    }

                }

                if (wordTemp != null)
                {                    

                    //No fim adiciona a palavra ao array List e incrementa o numero de palavras seleccionadas
                    if (!topWords.Contains(wordTemp))
                    {                        
                        topWords.Add(wordTemp);
                        count++;
                    }

                    else Console.Out.WriteLine("Something wrong is happening.. -> " + wordTemp.Concept);
                }
            }

            return topWords;
        }

        public List<Tag> mostRankWords(Hashtable keywords, int top)
        {
            //Adicona as palavras com maior rank num array List (algoritmo pesadito..)
            List<Tag> topWords = new List<Tag>();            

            int count = 1;
            //Enquanto ainda não tiver seleccionado as top tags mais ocorridas,
            //ou se a hashtable ficou sem keys antes de se chegar às 10 tags
            while ((count <= top && count < keywords.Count) && keywords.Count != 0)
            {
                int mostRanked = 1;
                Tag wordTemp = null;

                //percorre a hashtable
                foreach (String word in keywords.Keys)
                {

                    int actualRanked = ((Tag)keywords[word]).Occurence;

                    //Se a key actual tiver maior ranking, passa a key para uma variável, e actualiza o maior ranking encontrado até agora
                    if (actualRanked >= mostRanked)
                    {
                        if (!topWords.Contains((Tag)keywords[word]))
                        {
                            wordTemp = (Tag)keywords[word];
                            mostRanked = actualRanked;
                        }
                    }

                }

                if (wordTemp != null)
                {                    

                    //No fim adiciona a palavra ao array List e incrementa o numero de palavras seleccionadas
                    if (!topWords.Contains(wordTemp))
                    {                        
                        topWords.Add(wordTemp);
                        count++;
                    }

                    else Console.Out.WriteLine("Something wrong is happening.. -> " + wordTemp.Concept);
                }
            }

            return topWords;
        }
        /// <summary>
        /// Processa os vários métodos desde a extracção de palavras do html document até as palavras fazerem match com a ontologia
        /// </summary>
        /// <param name="localOntology">ontologia local do smart browser</param>
        public void getTags(Ontology.Ontology localOntology)
        {
            //Tira conceitos importantes dos sites. Considera-se conceitos importantes,
            //as palavras que estão em secções de destaque do documento
            List<Tag> buzzWords = this.findHTMLBuzzWords();            

            //Elimina as stop words da lista de conceitos
            buzzWords = readStopWords(buzzWords);

            //Novo método. Dado que o método anterior nos deu muito tipo de informação, e que juntamente com as transformações às conceitos,
            //ficamos com imensos tipos de conceitos, considerou-se que o melhor era fazer um top ranking de ocorrências dos conceitos no documento.
            this.rankByOccurence(buzzWords);

            //List<Tag> rankTags = this.mostRankWords(this.keywords);

            //Normaliza as tags, transformando advérbios, adjectivos ( etc.. ) em nomes

            List<Tag> nounTags = new List<Tag>();

            foreach (Tag tag in buzzWords)
            {
                nounTags.AddRange(this.normalizeTag(tag));
            }

            //Para que não haja palavras repetidas
            foreach (Tag tag in nounTags)
                if (!buzzWords.Contains(tag))
                    buzzWords.Add(tag);            
           
            matchTags(buzzWords, localOntology);           
        }

        /// <summary>
        /// Normaliza a palavra, transformando as várias terminações possíveis em terminações típicas em nomes
        /// </summary>
        /// <param name="WordText"></param>
        /// <returns></returns>
        public List<Tag> normalizeTag(Tag tag)
        {

            String[] NounSES = { "ses", "s" };
            String[] NounS = { "s", "" };
            String[] NounXES = { "xes", "x" };
            String[] NounZES = { "zes", "z" };
            String[] NounCHES = { "ches", "ch" };
            String[] NounSHES = { "shes", "sh" };

            String[] VerbIES = { "ies", "y" };
            String[] VerbES = { "es", "e", "" };
            String[] VerbED = { "ed", "e", "" };
            String[] VerbING = { "ing", "e", "" };

            String[] AdjER = { "er", "e", "" };
            String[] AdjEST = { "est", "e", "e" };

            List<Tag> Words = new List<Tag>();            

            if (tag.Concept.ToLower().EndsWith(NounSES[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, NounSES[0], NounSES[1]));
                tagTransform.Occurence = tag.Occurence;
                if(!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(NounS[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, NounS[0], NounS[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);               
            }

            if (tag.Concept.ToLower().EndsWith(NounXES[0]))
            {
               Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, NounXES[0], NounXES[1]));
               tagTransform.Occurence = tag.Occurence;
               if (!Words.Contains(tagTransform))
                   Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(NounZES[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, NounZES[0], NounZES[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(NounCHES[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, NounCHES[0], NounCHES[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(NounSHES[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, NounSHES[0], NounSHES[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(NounSHES[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, NounSHES[0], NounSHES[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(VerbES[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, VerbES[0], VerbES[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);

                tagTransform = new Tag(null, GetTransformation(tag.Concept, VerbES[0], VerbES[2]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(VerbED[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, VerbED[0], VerbED[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);

                tagTransform = new Tag(null, GetTransformation(tag.Concept, VerbED[0], VerbED[2]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(VerbIES[0]))
            {

                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, VerbIES[0], VerbIES[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);         
            }

            if (tag.Concept.ToLower().EndsWith(VerbING[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, VerbING[0], VerbING[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);

                tagTransform = new Tag(null, GetTransformation(tag.Concept, VerbING[0], VerbING[2]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(AdjER[0]))
            {
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, AdjER[0], AdjER[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);

                tagTransform = new Tag(null, GetTransformation(tag.Concept, AdjER[0], AdjER[2]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }

            if (tag.Concept.ToLower().EndsWith(AdjEST[0]))
            {               
                Tag tagTransform = new Tag(null, GetTransformation(tag.Concept, AdjEST[0], AdjEST[1]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);

                tagTransform = new Tag(null, GetTransformation(tag.Concept, AdjEST[0], AdjEST[2]));
                tagTransform.Occurence = tag.Occurence;
                if (!Words.Contains(tagTransform))
                    Words.Add(tagTransform);
            }            
            
            return Words;
        }

        private String GetTransformation(String WordText, String Suffix, String Ending)
        {
            WordText = WordText.Remove((WordText.Length - 1) - (Suffix.Length - 1));
            WordText = WordText + Ending;

            return WordText;
        }
       
    }
}
